Resumen Resultados V2

En este notebook aplicamos los test estadísticos y el algoritmo decisión multicriterio a aquellos modelos que pensamos que sufren de menos overfitting.

import pandas as pd
from pandas import read_csv
import seaborn as sns

from sklearn.model_selection import TimeSeriesSplit
from sklearn.model_selection import train_test_split

import joblib as joblib

import matplotlib.pyplot as plt

import scikit_posthocs as scp
import scipy.stats as stats
import operator
import numpy as np
import statsmodels as st
import pingouin as pg

import math
import sklearn.metrics as sm

from plotly.subplots import make_subplots
import plotly.express as px
import plotly
import plotly.graph_objects as go
import flask

C:\Users\Carmen\anaconda3\lib\site-packages\outdated\utils.py:14: OutdatedPackageWarning: The package outdated is out of date. Your version is 0.2.0, the latest is 0.2.1.
Set the environment variable OUTDATED_IGNORE=1 to disable these warnings.
  return warn(

Eliminando modelos que padecen overfitting

df_predictor_lag3_escalado = read_csv('../Datos_preprocesados/predictor_lag3_escalado.csv', encoding='latin-1', sep = ',', na_values = ['NaN', 'NaT'])
df_predictor_lag3_escalado = df_predictor_lag3_escalado.set_index('Fecha')

df_predictor_lag5_escalado = read_csv('../Datos_preprocesados/predictor_lag5_escalado.csv', encoding='latin-1', sep = ',', na_values = ['NaN', 'NaT'])
df_predictor_lag5_escalado = df_predictor_lag5_escalado.set_index('Fecha')

X_train_3, X_test_3, y_train_3, y_test_3 = train_test_split(df_predictor_lag3_escalado.drop(['Incidentes'], axis=1),
                                        df_predictor_lag3_escalado['Incidentes'], train_size = 0.8, test_size = 0.2, 
                                        random_state = 42, shuffle = False)

X_train_5, X_test_5, y_train_5, y_test_5 = train_test_split(df_predictor_lag5_escalado.drop(['Incidentes'], axis=1),
                                        df_predictor_lag5_escalado['Incidentes'], train_size = 0.8, test_size = 0.2, 
                                        random_state = 42, shuffle = False)

Cargamos los modelos

modelos = []

modelos.append(joblib.load('rf_lag3_escalado.pkl'))
modelos.append(joblib.load('rf_lag5_escalado.pkl'))

modelos.append(joblib.load('regresion_lineal_lag3_escalado.pkl'))
modelos.append(joblib.load('regresion_lineal_lag5_escalado.pkl'))

modelos.append(joblib.load('lasso_lag3_escalado.pkl'))
modelos.append(joblib.load('lasso_lag5_escalado.pkl'))

modelos.append(joblib.load('svr_lag3_escalado.pkl'))
modelos.append(joblib.load('svr_lag5_escalado.pkl'))

modelos.append(joblib.load('gbr_lag3_escalado.pkl'))
modelos.append(joblib.load('gbr_lag5_escalado.pkl'))

modelos.append(joblib.load('en_lag3_escalado.pkl'))
modelos.append(joblib.load('en_lag5_escalado.pkl'))

nombres = ['RF Lag3', 'RF Lag5', 'LR Lag3', 'LR Lag5', 'Lasso Lag3', 'Lasso Lag5', 'SVR Lag3', 'SVR Lag5', 'GBR Lag3', 'GBR Lag5', 'EN Lag3', 'EN Lag5']

Función que predice el instante t tanto en train como en test para cada modelo y muestra las gráficas

def pred_train_test(modelo, lag):
    
    if lag == 3:
        X_train = X_train_3
        y_train = y_train_3
        X_test = X_test_3
        y_test = y_test_3
    if lag == 5:
        X_train = X_train_5
        y_train = y_train_5
        X_test = X_test_5
        y_test = y_test_5
        
    fig = make_subplots(rows=1, cols=2)
    
    # Predicción en train
    y_pred_train_modelo = modelo.predict(X_train)
    # Predicción en test
    y_pred_test_modelo = modelo.predict(X_test)
    
    fig.add_trace(go.Scatter(x = y_train.index, y = y_pred_train_modelo, mode = 'lines', name = 'Train Real'), row = 1, col = 1)
    fig.add_trace(go.Scatter(x = y_train.index, y = y_train, mode = 'lines', name = 'Pred Train'), row = 1, col = 1)
    
    fig.add_trace(go.Scatter(x = y_test.index, y = y_pred_test_modelo, mode = 'lines', name = 'Test Real'), row = 1, col = 2)
    fig.add_trace(go.Scatter(x = y_test.index, y = y_test, mode = 'lines', name = 'Pred test'), row = 1, col = 2)
    
    fig.update_layout(showlegend = True, title_text = 'Predicciones Train VS Test ' + str(nombres[i]))
    
    flask.Markup(fig)
    
    fig.show("notebook")

for i in range(0, len(modelos)):
    nombre = nombres[i]
    # Si es numero par
    if i % 2 == 0:
        lag = 3
    else:
        lag = 5
    pred_train_test(modelos[i], lag)

De forma gráfica parece que los modelos que más overfitting hacen son Ranfom Forest y GBR (tanto para lag3 como para lag5). Ahora cogemos las medidas de error en la predicción t tanto para train como para test de todos los modelos. Aquellos que tengan una diferencia > 0.2 serán los que consideremos que están sobre ajustando los datos

Leemos las tablas de errores de RMSE y MAE tanto para train como para test

RMSE_train = pd.read_csv('../Datos_preprocesados/RMSE_errores_train.csv')
RMSE_test = pd.read_csv('../Datos_preprocesados/RMSE_errores_test.csv')
RMSE_train.drop(columns = "Pliegues", inplace = True)
RMSE_test.drop(columns = "Predicciones", inplace = True)

MAE_train = pd.read_csv('../Datos_preprocesados/MAE_errores_train.csv')
MAE_test = pd.read_csv('../Datos_preprocesados/MAE_errores_test.csv')
MAE_train.drop(columns = "Pliegues", inplace  =True)
MAE_test.drop(columns = "Predicciones", inplace = True)

CC_train = pd.read_csv('../Datos_preprocesados/CC_train.csv')
CC_train.drop(columns = "Pliegues", inplace = True)
CC_test = pd.read_csv('../Datos_preprocesados/CC_test.csv')
CC_test.drop(columns = "Predicciones", inplace = True)

Ahora cogemos las predicciones en t de ambas tablas, es decir, la primera fila de cada tabla

RMSE_train_t = RMSE_train.iloc[0]
RMSE_test_t = RMSE_test.iloc[0]

MAE_train_t = MAE_train.iloc[0]
MAE_test_t = MAE_test.iloc[0]

Calculamos las diferencias y aquellas que superen el valor de 0.2 serán los modelos que eliminaremos

RMSE_dif = RMSE_test_t - RMSE_train_t
MAE_dif = MAE_test_t - MAE_train_t

overfitting = []
for i in range(0, len(RMSE_dif)):
    if RMSE_dif[i] > 0.2 or MAE_dif[i] > 0.2:
        overfitting.append(i)

print(MAE_dif, RMSE_dif)

RF_lag3       0.131538
RF_lag5       0.128608
LR_lag3       0.057521
LR_lag5       0.065008
Lasso_lag3    0.082773
Lasso_lag5    0.072918
SVR_lag3      0.077377
SVR_lag5      0.062506
GBR_lag3      0.122226
GBR_lag5      0.120121
EN_lag3       0.083524
EN_lag5       0.074019
Name: 0, dtype: float64 RF_lag3       0.209750
RF_lag5       0.217747
LR_lag3       0.089960
LR_lag5       0.108105
Lasso_lag3    0.119223
Lasso_lag5    0.117825
SVR_lag3      0.114532
SVR_lag5      0.112392
GBR_lag3      0.200221
GBR_lag5      0.198579
EN_lag3       0.120435
EN_lag5       0.118698
Name: 0, dtype: float64

Eliminamos los modelos que han hecho overfitting

RMSE_train.drop(columns = RMSE_train.columns[overfitting], inplace=True)
RMSE_test.drop(columns = RMSE_test.columns[overfitting], inplace=True)
MAE_train.drop(columns = MAE_train.columns[overfitting], inplace=True)
MAE_test.drop(columns = MAE_test.columns[overfitting], inplace=True)
CC_train.drop(columns = CC_train.columns[overfitting], inplace=True)
CC_test.drop(columns = CC_test.columns[overfitting], inplace=True)

Tests estadísticos

Partimos de las tablas a las que hemos eliminado los modelos que sobre ajustan los datos y aplicamos los test estadísticos

def pairwise_test(df, parametric = True, method = 'bonferroni', decreasing = True, ties=True):
    # Función que devuelve la matriz de rankings.
    # El dataframe de entrada df debe tener los modelos en las columnas y 
    # las medidas en las filas. 
    # parametric = indica si se aplica un test paramétrico (ttest_rel) o no paramétrico (wilcoxon)
    # decreasing = indica el sentido en el que es mejor la medida utilizada. 
    # ties = indca si se incluyen los empates en el resultado final 
    
    if decreasing:
        relate = operator.lt
    else:
        relate = operator.gt
    
    if parametric:
        test = stats.ttest_rel
    else:
        test = stats.wilcoxon
        
    cols = df.columns
    pvalues = np.array([])
    significant = np.array([])
    ranks = pd.DataFrame({"wins":np.zeros(len(cols)),
                         "ties":np.zeros(len(cols)),
                         "losses":np.zeros(len(cols))})
    ranks.index= cols
    for index1 in range(len(cols)):
        for index2 in range(index1+1,len(cols)):
            T,p = test(df.iloc[:,index1],df.iloc[:,index2])
            pvalues = np.append(pvalues,p)
            
    significant,padjust,a1,a2 = st.stats.multitest.multipletests(pvalues,alpha = 0.05,method = method)

    
    pos = 0;
    for index1 in range(len(cols)):
        for index2 in range(index1+1,len(cols)):
            if not significant[pos]:
                ranks.iloc[index1,1] = ranks.iloc[index1,1]+1
                ranks.iloc[index2,1] = ranks.iloc[index2,1]+1
            else:  
                if relate(df.iloc[:,index1].mean(),df.iloc[:,index2].mean()):
                    ranks.iloc[index1,0] = ranks.iloc[index1,0]+1
                    ranks.iloc[index2,2] = ranks.iloc[index2,2]+1
                else:
                    ranks.iloc[index2,0] = ranks.iloc[index2,0]+1
                    ranks.iloc[index1,2] = ranks.iloc[index1,2]+1
            pos = pos + 1
                
    ranks["diff"] = ranks.wins-ranks.losses
    #ranks = ranks.sort_values(by="diff",ascending = False)
    if  not ties:
        ranks.drop('ties',axis = 'columns', inplace = True)
    return ranks.astype(int)

def tabla_wins_losses(criterio, w_l):
    #global tabla
    if criterio == 'CC':
        tabla = CC_train
        nemenyi = nemenyi_cc
    if criterio == 'MAE':
        tabla = MAE_train
        nemenyi = nemenyi_mae
    tam = len(tabla.columns)
    # Recorremos las columnas
    for j in range(tam):
        # Recorremos las filas evitando las ya leidas al ser la matriz simetrica
        for i in range(j+1,tam):
            # Si p-value < 0.05
            if nemenyi.iat[i, j] < 0.05:
                # Cogemos los modelos con diferencias significativas
                modelo1 = columnas[i]
                modelo2 = columnas[j]
                if criterio == 'CC':
                    # Tomamos su CC maximo entre todos los fold
                    cc_modelo1 = pd.Series(tabla[modelo1]).mean()
                    cc_modelo2 = pd.Series(tabla[modelo2]).mean()
                    # El modelo correspondiente al mayor CC se añade un Win y el otro un Loss
                    if cc_modelo1 > cc_modelo2:
                        # Añadimos un Win al modelo1
                        w_l.iat[i,0] = w_l.iat[i,0] + 1
                        # Añadimos un Loss al modelo2
                        w_l.iat[j,1] = w_l.iat[j,1] + 1
                    if cc_modelo1 < cc_modelo2:
                        # Añadimos un Win al modelo2
                        w_l.iat[j,0] = w_l.iat[j,0] + 1
                        # Añadimos un Loss al modelo1
                        w_l.iat[i,1] = w_l.iat[i,1] + 1     
                if criterio == 'MAE':
                    # Tomamos su MAE mínimo entre todos los fold
                    mae_modelo1 = pd.Series(tabla[modelo1]).mean()
                    mae_modelo2 = pd.Series(tabla[modelo2]).mean()
                    # El modelo correspondiente al mayor CC se añade un Win y el otro un Loss
                    if mae_modelo1 < mae_modelo2:
                        # Añadimos un Win al modelo1
                        w_l.iat[i,0] = w_l.iat[i,0] + 1
                        # Añadimos un Loss al modelo2
                        w_l.iat[j,1] = w_l.iat[j,1] + 1
                    if mae_modelo1 > mae_modelo2:
                        # Añadimos un Win al modelo2
                        w_l.iat[j,0] = w_l.iat[j,0] + 1
                        # Añadimos un Loss al modelo1
                        w_l.iat[i,1] = w_l.iat[i,1] + 1

RMSE

fig = px.box(RMSE_train, y = RMSE_train.columns)
flask.Markup(fig)
fig.show()

RMSE_train.apply(stats.shapiro)

	LR_lag3	LR_lag5	Lasso_lag3	Lasso_lag5	SVR_lag3	SVR_lag5	GBR_lag5	EN_lag3	EN_lag5
0	0.926488	0.894526	0.881280	0.884258	0.874022	0.886963	0.855404	0.880959	0.884850
1	0.414248	0.190580	0.134983	0.145985	0.111330	0.156691	0.067337	0.133844	0.148267

test = stats.friedmanchisquare(*[RMSE_train[col] for col in RMSE_train.columns])
test

FriedmanchisquareResult(statistic=17.73333333333335, pvalue=0.023317036199846892)

scp.posthoc_nemenyi_friedman(RMSE_train, melted=False)

	LR_lag3	LR_lag5	Lasso_lag3	Lasso_lag5	SVR_lag3	SVR_lag5	GBR_lag5	EN_lag3	EN_lag5
LR_lag3	1.000000	0.900000	0.900000	0.858844	0.900000	0.900000	0.900000	0.900000	0.900000
LR_lag5	0.900000	1.000000	0.759750	0.512019	0.900000	0.900000	0.900000	0.660658	0.561564
Lasso_lag3	0.900000	0.759750	1.000000	0.900000	0.351152	0.900000	0.561564	0.900000	0.900000
Lasso_lag5	0.858844	0.512019	0.900000	1.000000	0.149525	0.710204	0.301968	0.900000	0.900000
SVR_lag3	0.900000	0.900000	0.351152	0.149525	1.000000	0.900000	0.900000	0.257126	0.180790
SVR_lag5	0.900000	0.900000	0.900000	0.710204	0.900000	1.000000	0.900000	0.858844	0.759750
GBR_lag5	0.900000	0.900000	0.561564	0.301968	0.900000	0.900000	1.000000	0.460015	0.351152
EN_lag3	0.900000	0.660658	0.900000	0.900000	0.257126	0.858844	0.460015	1.000000	0.900000
EN_lag5	0.900000	0.561564	0.900000	0.900000	0.180790	0.759750	0.351152	0.900000	1.000000

No detecta diferencias significativas

rsme_ranks = pairwise_test(RMSE_train, parametric=False, decreasing=True, ties= False, method="holm")
rsme_ranks

	wins	losses	diff
LR_lag3	0	0	0
LR_lag5	0	0	0
Lasso_lag3	0	0	0
Lasso_lag5	0	0	0
SVR_lag3	0	0	0
SVR_lag5	0	0	0
GBR_lag5	0	0	0
EN_lag3	0	0	0
EN_lag5	0	0	0

MAE

fig = px.box(MAE_train, y = MAE_train.columns)
flask.Markup(fig)
fig.show()

MAE_train.apply(stats.shapiro)

	LR_lag3	LR_lag5	Lasso_lag3	Lasso_lag5	SVR_lag3	SVR_lag5	GBR_lag5	EN_lag3	EN_lag5
0	0.935417	0.912743	0.886039	0.917546	0.869829	0.946054	0.859454	0.894715	0.919158
1	0.503236	0.300369	0.152955	0.336945	0.099507	0.622112	0.075185	0.191507	0.349987

test = stats.friedmanchisquare(*[MAE_train[col] for col in MAE_train.columns])
test

FriedmanchisquareResult(statistic=21.54666666666668, pvalue=0.00582864737876288)

nemenyi_mae = scp.posthoc_nemenyi_friedman(MAE_train, melted=False)
nemenyi_mae

	LR_lag3	LR_lag5	Lasso_lag3	Lasso_lag5	SVR_lag3	SVR_lag5	GBR_lag5	EN_lag3	EN_lag5
LR_lag3	1.000000	0.90000	0.90000	0.900000	0.809296	0.900000	0.900000	0.900000	0.405206
LR_lag5	0.900000	1.00000	0.90000	0.900000	0.216940	0.900000	0.611110	0.900000	0.900000
Lasso_lag3	0.900000	0.90000	1.00000	0.900000	0.216940	0.900000	0.611110	0.900000	0.900000
Lasso_lag5	0.900000	0.90000	0.90000	1.000000	0.180790	0.900000	0.561564	0.900000	0.900000
SVR_lag3	0.809296	0.21694	0.21694	0.180790	1.000000	0.900000	0.900000	0.180790	0.005415
SVR_lag5	0.900000	0.90000	0.90000	0.900000	0.900000	1.000000	0.900000	0.900000	0.301968
GBR_lag5	0.900000	0.61111	0.61111	0.561564	0.900000	0.900000	1.000000	0.561564	0.049862
EN_lag3	0.900000	0.90000	0.90000	0.900000	0.180790	0.900000	0.561564	1.000000	0.900000
EN_lag5	0.405206	0.90000	0.90000	0.900000	0.005415	0.301968	0.049862	0.900000	1.000000

Detecta diferencias en EN_Lag5 VS GBR_Lag5 y EN_Lag5 VS SVR_Lag3

mae_ranks = pairwise_test(MAE_train, parametric=False, decreasing=True, ties= False, method="holm")
mae_ranks

	wins	losses	diff
LR_lag3	0	0	0
LR_lag5	0	0	0
Lasso_lag3	0	0	0
Lasso_lag5	0	0	0
SVR_lag3	0	0	0
SVR_lag5	0	0	0
GBR_lag5	0	0	0
EN_lag3	0	0	0
EN_lag5	0	0	0

columnas = []

for c in nemenyi_mae.columns:
    columnas.append(c)

modelos = MAE_train.columns

ceros = [0]*len(modelos)

d = {'Wins': ceros, 'Losses': ceros}

wins_losses_mae = pd.DataFrame(data = d)
wins_losses_mae = wins_losses_mae.set_index(modelos)

tabla_wins_losses('MAE', wins_losses_mae)
wins_losses_mae

	Wins	Losses
LR_lag3	0	0
LR_lag5	0	0
Lasso_lag3	0	0
Lasso_lag5	0	0
SVR_lag3	0	1
SVR_lag5	0	0
GBR_lag5	0	1
EN_lag3	0	0
EN_lag5	2	0

CC

fig = px.box(CC_train, y = CC_train.columns)
flask.Markup(fig)
fig.show()

CC_train.apply(stats.shapiro)

	LR_lag3	LR_lag5	Lasso_lag3	Lasso_lag5	SVR_lag3	SVR_lag5	GBR_lag5	EN_lag3	EN_lag5
0	0.958121	0.887266	0.957226	0.912596	0.934377	0.846537	0.716735	0.948635	0.923018
1	0.764250	0.157936	0.753839	0.299302	0.492296	0.052824	0.001406	0.652372	0.382816

test = stats.friedmanchisquare(*[CC_train[col] for col in CC_train.columns])
test

FriedmanchisquareResult(statistic=34.2092050209205, pvalue=3.7229147886482146e-05)

nemenyi_cc = scp.posthoc_nemenyi_friedman(CC_train, melted=False)
nemenyi_cc

	LR_lag3	LR_lag5	Lasso_lag3	Lasso_lag5	SVR_lag3	SVR_lag5	GBR_lag5	EN_lag3	EN_lag5
LR_lag3	1.000000	0.858844	0.900000	0.635884	0.900000	0.586338	0.001000	0.900000	0.858844
LR_lag5	0.858844	1.000000	0.784522	0.900000	0.900000	0.900000	0.038985	0.900000	0.900000
Lasso_lag3	0.900000	0.784522	1.000000	0.561564	0.900000	0.512019	0.001000	0.900000	0.784522
Lasso_lag5	0.635884	0.900000	0.561564	1.000000	0.900000	0.900000	0.110517	0.710204	0.900000
SVR_lag3	0.900000	0.900000	0.900000	0.900000	1.000000	0.900000	0.013263	0.900000	0.900000
SVR_lag5	0.586338	0.900000	0.512019	0.900000	0.900000	1.000000	0.135069	0.660658	0.900000
GBR_lag5	0.001000	0.038985	0.001000	0.110517	0.013263	0.135069	1.000000	0.001000	0.038985
EN_lag3	0.900000	0.900000	0.900000	0.710204	0.900000	0.660658	0.001000	1.000000	0.900000
EN_lag5	0.858844	0.900000	0.784522	0.900000	0.900000	0.900000	0.038985	0.900000	1.000000

Detecta diferencias entre Lasso_Lag3 VS RF_Lag5, SVR_Lag3 VS

cc_ranks = pairwise_test(CC_train, parametric=False, decreasing=False, ties= False, method="holm")
cc_ranks

C:\Users\Carmen\anaconda3\lib\site-packages\scipy\stats\morestats.py:2957: UserWarning:

Exact p-value calculation does not work if there are ties. Switching to normal approximation.

C:\Users\Carmen\anaconda3\lib\site-packages\scipy\stats\morestats.py:2971: UserWarning:

Sample size too small for normal approximation.

	wins	losses	diff
LR_lag3	0	0	0
LR_lag5	0	0	0
Lasso_lag3	0	0	0
Lasso_lag5	0	0	0
SVR_lag3	0	0	0
SVR_lag5	0	0	0
GBR_lag5	0	0	0
EN_lag3	0	0	0
EN_lag5	0	0	0

columnas = []

for c in nemenyi_cc.columns:
    columnas.append(c)

modelos = CC_train.columns

ceros = [0]*len(modelos)

d = {'Wins': ceros, 'Losses': ceros}

wins_losses_cc = pd.DataFrame(data = d)
wins_losses_cc = wins_losses_cc.set_index(modelos)

tabla_wins_losses('CC', wins_losses_cc)
wins_losses_cc

	Wins	Losses
LR_lag3	0	1
LR_lag5	0	1
Lasso_lag3	0	1
Lasso_lag5	0	0
SVR_lag3	0	1
SVR_lag5	0	0
GBR_lag5	6	0
EN_lag3	0	1
EN_lag5	0	1

Generamos la tabla completa y ordenamos

wins_losses = wins_losses_cc + wins_losses_mae
wins_losses['Diff'] = wins_losses['Wins'] - wins_losses['Losses']
wins_losses

	Wins	Losses	Diff
LR_lag3	0	1	-1
LR_lag5	0	1	-1
Lasso_lag3	0	1	-1
Lasso_lag5	0	0	0
SVR_lag3	0	2	-2
SVR_lag5	0	0	0
GBR_lag5	6	1	5
EN_lag3	0	1	-1
EN_lag5	2	1	1

wins_losses_rank = wins_losses.sort_values('Diff', ascending = False)
wins_losses_rank

	Wins	Losses	Diff
GBR_lag5	6	1	5
EN_lag5	2	1	1
Lasso_lag5	0	0	0
SVR_lag5	0	0	0
LR_lag3	0	1	-1
LR_lag5	0	1	-1
Lasso_lag3	0	1	-1
EN_lag3	0	1	-1
SVR_lag3	0	2	-2

Algoritmo de decisión multicriterio